{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77e48e15",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import traceback\n",
    "import json\n",
    "import math\n",
    "from functools import partial\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.stats import spearmanr, kendalltau\n",
    "from collections import Counter\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b12b018a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!tree results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3c8897ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = \"./_answers/results_gemini_0shot\"\n",
    "HUMAN_RESULTS = \"./_human_annotations/ImagenHub_human_eval_results\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "26dbf735",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python count_entries.py results_gemini_0shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d3426564",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigfig(number, sigfigs=4, digit_mode=True):\n",
    "    \"\"\"\n",
    "    Convert a number to its significant figure representation.\n",
    "    \n",
    "    Args:\n",
    "        number (float/list): Number or list of numbers to convert.\n",
    "        sigfigs (int, optional): Number of significant figures to keep. Defaults to 4.\n",
    "        digit_mode (bool, optional): If set to True, will use the digit mode for formatting. Defaults to True.\n",
    "        \n",
    "    Returns:\n",
    "        float/list: Number(s) in their significant figure representation.\n",
    "    \"\"\"\n",
    "    if digit_mode:\n",
    "        string_mode = '{:#.{sigfigs}f}'\n",
    "    else:\n",
    "        string_mode = '{:#.{sigfigs}g}'\n",
    "    if isinstance(number, list):\n",
    "        new_numbers = []\n",
    "        for num in number:\n",
    "            new_num = string_mode.format(num, sigfigs=sigfigs)\n",
    "            new_numbers.append(float(new_num))\n",
    "        return new_numbers\n",
    "    else:\n",
    "        return float(string_mode.format(number, sigfigs=sigfigs))\n",
    "\n",
    "def map_to_nearest_higher(number, target_numbers=[0.0, 0.17, 0.33, 0.5, 0.67, 0.83, 1.0], not_mapping=True):\n",
    "    \"\"\"\n",
    "    Maps the given number to the nearest higher number in the target_numbers list.\n",
    "    If the number is higher than the highest in the list, return the highest number.\n",
    "\n",
    "    :param number: A float number between 0.0 and 1.0.\n",
    "    :param target_numbers: A list of numbers to which the given number should be mapped.\n",
    "    :return: The nearest higher number in the target_numbers list to the given number.\n",
    "    \"\"\"\n",
    "    if not_mapping:\n",
    "        if number > 1.0:\n",
    "            return 1.0\n",
    "        if number < 0.0:\n",
    "            return 0.0\n",
    "        return number\n",
    "    \n",
    "    # Sort the target numbers just in case they are not sorted\n",
    "    target_numbers = sorted(target_numbers)\n",
    "\n",
    "    # Find the nearest higher number\n",
    "    for target in target_numbers:\n",
    "        if target >= number:\n",
    "            return target\n",
    "    return target_numbers[-1]  # Return the maximum if no higher number is found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2569444b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_json_data(json_data):\n",
    "    \"\"\"\n",
    "    Processes the provided JSON data (in dictionary form) to retain only the 'score' values for each entry.\n",
    "\n",
    "    :param json_data: A dictionary representing the JSON data.\n",
    "    :return: A dictionary with only the 'score' values.\n",
    "    \"\"\"\n",
    "    processed_data = {key: value['score'] for key, value in json_data.items()}\n",
    "\n",
    "    return processed_data\n",
    "\n",
    "def read_json_files(root_dir=ROOT):\n",
    "    \"\"\"\n",
    "    Reads all JSON files in the specified directory structure and returns their contents.\n",
    "\n",
    "    :param root_dir: The root directory to start the search from.\n",
    "    :return: A dictionary with file paths as keys and their content as values.\n",
    "    \"\"\"\n",
    "    json_contents = {}\n",
    "    \n",
    "    # Walk through the directory\n",
    "    for dirpath, dirnames, filenames in os.walk(root_dir):\n",
    "        for file in filenames:\n",
    "            if file.endswith('.json'):\n",
    "                # Construct full file path\n",
    "                file_path = os.path.join(dirpath, file)\n",
    "                \n",
    "                # Read the JSON file\n",
    "                with open(file_path, 'r') as json_file:\n",
    "                    data = json.load(json_file)\n",
    "                    data = process_json_data(data)\n",
    "                    json_contents[file_path] = data\n",
    "\n",
    "    return json_contents\n",
    "\n",
    "\n",
    "def read_json(result_folder, task, modelname, filename):\n",
    "    # Construct the full path to the JSON file\n",
    "    file_path = os.path.join(result_folder, task, modelname, filename)\n",
    "    \n",
    "    try:\n",
    "        # Read the JSON file\n",
    "        with open(file_path, 'r') as file:\n",
    "            data = json.load(file)\n",
    "    except:\n",
    "        return None\n",
    "    \n",
    "    return data\n",
    "\n",
    "def grab_gpt4v_processed(task, modelname, filename, result_folder=ROOT, dumb_way=False):\n",
    "    if dumb_way:\n",
    "        out = read_json_files()\n",
    "        return out[f'{result_folder}/{task}/{modelname}/{filename}.json']\n",
    "    else:\n",
    "        data = read_json(result_folder, task, modelname, f\"{filename}.json\")\n",
    "        if data is not None:\n",
    "            data = process_json_data(data)\n",
    "        return data\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "877b8011",
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterate_all_results(root=ROOT):\n",
    "    task_list = os.listdir(root) \n",
    "    for task in task_list:\n",
    "        print(\"===============>\", task)\n",
    "        model_list = os.listdir(os.path.join(root,task))\n",
    "        for model in model_list:\n",
    "            print(\"======>\", model, \": \")\n",
    "            SC_results = grab_gpt4v_processed(task, model, \"SC\")\n",
    "            PQ_results = grab_gpt4v_processed(task, model, \"PQ\")\n",
    "            \n",
    "            SC_results, PQ_results = align_dicts(SC_results, PQ_results)\n",
    "            \n",
    "            \n",
    "            SC_list = parse_scores_to_list(SC_results)\n",
    "            print(\"Length: \",len(SC_list))\n",
    "            SC_mean, SC_std  = get_statistics(SC_list)\n",
    "            print(\"SC | mean: \", sigfig(SC_mean), \" std: \", sigfig(SC_std))\n",
    "\n",
    "            PQ_list = parse_scores_to_list(PQ_results)\n",
    "            PQ_mean, PQ_std = get_statistics(PQ_list)\n",
    "            print(\"PQ | mean: \", sigfig(PQ_mean), \" std: \", sigfig(PQ_std))\n",
    "            \n",
    "            O_list = get_O_from_two_lists(SC_list, PQ_list)\n",
    "            O_mean, O_std = get_statistics(O_list, False)\n",
    "            print(\"O | mean: \", sigfig(O_mean), \" std: \", sigfig(O_std))\n",
    "\n",
    "def align_dicts(dict1, dict2):\n",
    "    \"\"\"\n",
    "    Filter out both dict1 and dict2 to hold the common keys in the same order.\n",
    "    \"\"\"\n",
    "    # Find the common keys\n",
    "    common_keys = set(dict1.keys()) & set(dict2.keys())\n",
    "\n",
    "    # Filter and order dict1 and dict2 using the common keys\n",
    "    dict1_aligned = {key: dict1[key] for key in sorted(common_keys)}\n",
    "    dict2_aligned = {key: dict2[key] for key in sorted(common_keys)}\n",
    "\n",
    "    return dict1_aligned, dict2_aligned\n",
    "\n",
    "def parse_scores_to_list(data_dict):\n",
    "    score_list = []\n",
    "    for scores in data_dict.values():\n",
    "        score_list.append(scores)\n",
    "        #print(scores)\n",
    "\n",
    "    return score_list\n",
    "\n",
    "def preprocess(_list):\n",
    "    temp_list = []\n",
    "    for scores in _list:\n",
    "        if isinstance(scores, (int, float)):\n",
    "            temp_list.append(map_to_nearest_higher(scores/10.0))\n",
    "        else:\n",
    "            scores = [int(score) for score in scores]\n",
    "            temp_list.append(map_to_nearest_higher(min(scores)/10.0))\n",
    "    return temp_list\n",
    "    #return [min(scores)/10.0 for scores in _list]\n",
    "    \n",
    "def get_O_from_two_dicts_gpt4v(SC_dict, PQ_dict, normalize=True):\n",
    "    return_dict = {}\n",
    "    for key in SC_dict.keys():\n",
    "        if isinstance(SC_dict[key], (int, float)):\n",
    "            SC = map_to_nearest_higher(SC_dict[key]/10.0) if normalize else min(SC_dict[key])\n",
    "        else:\n",
    "            SC_dict[key] = [int(score) for score in SC_dict[key]]\n",
    "            SC = map_to_nearest_higher(min(SC_dict[key])/10.0) if normalize else min(SC_dict[key])\n",
    "\n",
    "        if isinstance(PQ_dict[key], (int, float)):\n",
    "            PQ = map_to_nearest_higher(PQ_dict[key]/10.0) if normalize else min(PQ_dict[key])\n",
    "        else:\n",
    "            PQ_dict[key] = [int(score) for score in PQ_dict[key]]\n",
    "            PQ = map_to_nearest_higher(min(PQ_dict[key])/10.0) if normalize else min(PQ_dict[key])\n",
    "        return_dict[key] = math.sqrt(SC * PQ)\n",
    "    return return_dict\n",
    "\n",
    "def get_O_from_two_lists(SC_list, PQ_list, preprocess_fn=preprocess):\n",
    "    \"\"\"\n",
    "    Compute the geometric mean of two lists on an instance level.\n",
    "\n",
    "    Parameters:\n",
    "    list1 (list): The first list of numbers.\n",
    "    list2 (list): The second list of numbers.\n",
    "\n",
    "    Returns:\n",
    "    list: A list containing the geometric means of corresponding elements from list1 and list2.\n",
    "    \"\"\"\n",
    "    # Check if both lists are of the same length\n",
    "    if len(SC_list) != len(PQ_list):\n",
    "        raise ValueError(\"Both lists must be of the same length\")\n",
    "    \n",
    "    if(preprocess_fn):\n",
    "        SC_list = preprocess(SC_list)\n",
    "        PQ_list = preprocess(PQ_list)\n",
    "    \n",
    "    # Calculate the geometric mean for each pair of elements\n",
    "    return [math.sqrt(x * y) for x, y in zip(SC_list, PQ_list)]\n",
    "\n",
    "\n",
    "def get_statistics(score_list, preprocess=True):\n",
    "    new_score_list = []\n",
    "    if preprocess:\n",
    "        for scores in score_list:\n",
    "            if isinstance(scores, (int, float)):\n",
    "                score = scores\n",
    "            else:\n",
    "                scores = [int(val) for val in scores]\n",
    "                score = min(scores)\n",
    "            #score = map_values(score)\n",
    "            new_score_list.append(map_to_nearest_higher(score/10.0))\n",
    "        score_list = new_score_list\n",
    "    mean = np.mean(score_list)\n",
    "    std = np.std(score_list)\n",
    "    return mean, std\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "25d9d3bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_scores(root=ROOT, per_model=False):\n",
    "    task_list = os.listdir(root) \n",
    "    for task in task_list:\n",
    "        print(\"===============>\", task)\n",
    "        model_list = os.listdir(os.path.join(root,task))\n",
    "        task_counter_SC = Counter()\n",
    "        task_counter_PQ = Counter()\n",
    "        task_counter_O = Counter()\n",
    "        \n",
    "        for model in model_list:\n",
    "            print(\"======>\", model, \": \")\n",
    "            SC_results = grab_gpt4v_processed(task, model, \"SC\")\n",
    "            PQ_results = grab_gpt4v_processed(task, model, \"PQ\")\n",
    "            \n",
    "            SC_results, PQ_results = align_dicts(SC_results, PQ_results)\n",
    "            \n",
    "            SC_list = parse_scores_to_list(SC_results)\n",
    "            PQ_list = parse_scores_to_list(PQ_results)\n",
    "            O_list = get_O_from_two_lists(SC_list, PQ_list)\n",
    "            print(\"SC list | \", sort_counter_keys(Counter(to_one_score_list(SC_list))))\n",
    "            print(\"PQ list | \", sort_counter_keys(Counter(to_one_score_list(PQ_list))))\n",
    "            print(\"O list | \", sort_counter_keys(correct_keys_and_sum(Counter(O_list))))\n",
    "            if per_model:\n",
    "                plot_counter_bar_chart(sort_counter_keys(Counter(to_one_score_list(SC_list))), \"red\", title=f\"{task} | {model}(SC)\")\n",
    "                plot_counter_bar_chart(sort_counter_keys(Counter(to_one_score_list(PQ_list))), \"blue\", title=f\"{task} | {model}(PQ)\")\n",
    "                plot_counter_bar_chart(correct_keys_and_sum(sort_counter_keys(correct_keys_and_sum(Counter(O_list)))), \"purple\", title=f\"{task} | {model}(O)\")\n",
    "            task_counter_SC += Counter(sort_counter_keys(sort_counter_keys(Counter(to_one_score_list(SC_list)))))\n",
    "            task_counter_PQ += Counter(sort_counter_keys(sort_counter_keys(Counter(to_one_score_list(PQ_list)))))\n",
    "            task_counter_O += Counter(sort_counter_keys(sort_counter_keys(Counter(O_list))))\n",
    "        \n",
    "        if not per_model:\n",
    "            plot_counter_bar_chart(task_counter_SC, \"red\", title=f\"{task} (SC)\")\n",
    "            plot_counter_bar_chart(task_counter_PQ, \"blue\", title=f\"{task} (PQ)\")\n",
    "            plot_counter_bar_chart(correct_keys_and_sum(task_counter_O), \"purple\", title=f\"{task} (O), nearest 0.1\")\n",
    "\n",
    "def analyze_scores_SC(root=ROOT, per_model=False):\n",
    "    task_list = os.listdir(root) \n",
    "    for task in task_list:\n",
    "        print(\"===============>\", task)\n",
    "        model_list = os.listdir(os.path.join(root,task))\n",
    "        task_counter_SC = Counter()\n",
    "        \n",
    "        for model in model_list:\n",
    "            print(\"======>\", model, \": \")\n",
    "            SC_results = grab_gpt4v_processed(task, model, \"SC\")\n",
    "            if SC_results is None:\n",
    "                print(f\"{task} | {model} not found\")\n",
    "                continue\n",
    "            SC_list = parse_scores_to_list(SC_results)\n",
    "            print(\"SC list | \", sort_counter_keys(Counter(to_one_score_list(SC_list))))\n",
    "            if per_model:\n",
    "                plot_counter_bar_chart(sort_counter_keys(Counter(to_one_score_list(SC_list))), \"red\", title=f\"{task} | {model}(SC)\")\n",
    "            task_counter_SC += Counter(sort_counter_keys(sort_counter_keys(Counter(to_one_score_list(SC_list)))))\n",
    "        \n",
    "        if not per_model:\n",
    "            plot_counter_bar_chart(task_counter_SC, \"red\", title=f\"{task} (SC)\")\n",
    "            \n",
    "def sort_counter_keys(counter):\n",
    "    # Sorting the counter by its keys and creating a sorted dictionary\n",
    "    sorted_dict = {k: counter[k] for k in sorted(counter)}\n",
    "\n",
    "    return sorted_dict\n",
    "\n",
    "def to_one_score_list(scores_list):\n",
    "    temp_list = []\n",
    "    for scores in scores_list:\n",
    "        if isinstance(scores, (int, float)):\n",
    "            temp_list.append(map_to_nearest_higher(scores/10.0))\n",
    "        else:\n",
    "            int_scores = []\n",
    "            # Iterate through each element in the list\n",
    "            for item in scores:\n",
    "                # Convert the item to an integer and append it to the int_list\n",
    "                int_scores.append(int(item))\n",
    "\n",
    "            temp_list.append(map_to_nearest_higher(min(int_scores)/10.0))\n",
    "    return temp_list\n",
    "\n",
    "def correct_keys_and_sum(counter):\n",
    "    \"\"\"\n",
    "    Adjusts the keys of a Counter object to the nearest 0.1 value and sums up the counts.\n",
    "\n",
    "    Parameters:\n",
    "    counter (Counter): The Counter object to be adjusted.\n",
    "\n",
    "    Returns:\n",
    "    Counter: The adjusted Counter object.\n",
    "    \"\"\"\n",
    "    new_counter = Counter()\n",
    "    for key, value in counter.items():\n",
    "        # Rounding the key to the nearest 0.1\n",
    "        corrected_key = round(key * 10) / 10\n",
    "        # Adding the count to the corrected key in the new counter\n",
    "        new_counter[corrected_key] += value\n",
    "    return new_counter\n",
    "\n",
    "def plot_counter_bar_chart(counter, color=\"blue\", bar_width=0.09, title=\"\", alpha=1.0):\n",
    "    \"\"\"\n",
    "    Plots a bar chart based on a Counter object.\n",
    "\n",
    "    Parameters:\n",
    "    counter (Counter): The Counter object to be plotted.\n",
    "    \"\"\"\n",
    "    # Extracting keys and values from the Counter object\n",
    "    keys = list(counter.keys())\n",
    "    values = list(counter.values())\n",
    "\n",
    "    # Plotting the bar chart\n",
    "    plt.figure(figsize=(5, 3))\n",
    "    plt.bar(keys, values, color=color, width=bar_width, alpha=alpha)\n",
    "    plt.xlabel('Score')\n",
    "    plt.ylabel('Counts')\n",
    "    plt.xticks(np.arange(0, 1.1, 0.1))\n",
    "    plt.title(title)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "baeb62bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#analyze_scores_SC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "de4a2c46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============> ImagenHub_Multi-Concept_IC\n",
      "======> TextualInversion : \n",
      "Length:  102\n",
      "SC | mean:  0.1176  std:  0.1712\n",
      "PQ | mean:  0.6206  std:  0.1301\n",
      "O | mean:  0.1851  std:  0.1888\n",
      "======> DreamBooth : \n",
      "Length:  102\n",
      "SC | mean:  0.1392  std:  0.1926\n",
      "PQ | mean:  0.6147  std:  0.1562\n",
      "O | mean:  0.2211  std:  0.1931\n",
      "======> CustomDiffusion : \n",
      "Length:  102\n",
      "SC | mean:  0.2167  std:  0.262\n",
      "PQ | mean:  0.6814  std:  0.144\n",
      "O | mean:  0.2977  std:  0.2495\n",
      "===============> ImagenHub_Text-Guided_IE\n",
      "======> Prompt2prompt : \n",
      "Length:  179\n",
      "SC | mean:  0.4084  std:  0.4562\n",
      "PQ | mean:  0.505  std:  0.1304\n",
      "O | mean:  0.307  std:  0.3334\n",
      "======> CycleDiffusion : \n",
      "Length:  179\n",
      "SC | mean:  0.3067  std:  0.4059\n",
      "PQ | mean:  0.4626  std:  0.1585\n",
      "O | mean:  0.2259  std:  0.2814\n",
      "======> SDEdit : \n",
      "Length:  179\n",
      "SC | mean:  0.3704  std:  0.4537\n",
      "PQ | mean:  0.5313  std:  0.1269\n",
      "O | mean:  0.2819  std:  0.3345\n",
      "======> DiffEdit : \n",
      "Length:  179\n",
      "SC | mean:  0.1615  std:  0.321\n",
      "PQ | mean:  0.3642  std:  0.1504\n",
      "O | mean:  0.1272  std:  0.2243\n",
      "======> MagicBrush : \n",
      "Length:  179\n",
      "SC | mean:  0.6263  std:  0.4325\n",
      "PQ | mean:  0.4855  std:  0.1375\n",
      "O | mean:  0.4544  std:  0.3143\n",
      "======> Text2Live : \n",
      "Length:  179\n",
      "SC | mean:  0.0771  std:  0.2472\n",
      "PQ | mean:  0.4933  std:  0.1385\n",
      "O | mean:  0.0571  std:  0.1722\n",
      "======> Pix2PixZero : \n",
      "Length:  179\n",
      "SC | mean:  0.1559  std:  0.3379\n",
      "PQ | mean:  0.5726  std:  0.1437\n",
      "O | mean:  0.1357  std:  0.2723\n",
      "======> InstructPix2Pix : \n",
      "Length:  179\n",
      "SC | mean:  0.4318  std:  0.4491\n",
      "PQ | mean:  0.5011  std:  0.1434\n",
      "O | mean:  0.3249  std:  0.3294\n",
      "===============> ImagenHub_Control-Guided_IG\n",
      "======> UniControl : \n",
      "Length:  150\n",
      "SC | mean:  0.4047  std:  0.373\n",
      "PQ | mean:  0.5093  std:  0.1494\n",
      "O | mean:  0.3626  std:  0.2781\n",
      "======> ControlNet : \n",
      "Length:  150\n",
      "SC | mean:  0.3793  std:  0.343\n",
      "PQ | mean:  0.4387  std:  0.1315\n",
      "O | mean:  0.333  std:  0.2441\n",
      "===============> ImagenHub_Mask-Guided_IE\n",
      "======> Glide : \n",
      "Length:  179\n",
      "SC | mean:  0.2223  std:  0.3838\n",
      "PQ | mean:  0.4877  std:  0.1297\n",
      "O | mean:  0.1649  std:  0.2674\n",
      "======> BlendedDiffusion : \n",
      "Length:  179\n",
      "SC | mean:  0.3575  std:  0.4126\n",
      "PQ | mean:  0.4017  std:  0.1305\n",
      "O | mean:  0.253  std:  0.2794\n",
      "======> SDInpaint : \n",
      "Length:  179\n",
      "SC | mean:  0.4441  std:  0.4404\n",
      "PQ | mean:  0.4341  std:  0.1492\n",
      "O | mean:  0.3186  std:  0.3031\n",
      "======> SDXLInpaint : \n",
      "Length:  179\n",
      "SC | mean:  0.5721  std:  0.4628\n",
      "PQ | mean:  0.5056  std:  0.134\n",
      "O | mean:  0.4192  std:  0.334\n",
      "===============> ImagenHub_Subject-Driven_IG\n",
      "======> TextualInversion : \n",
      "Length:  150\n",
      "SC | mean:  0.168  std:  0.2839\n",
      "PQ | mean:  0.5807  std:  0.1526\n",
      "O | mean:  0.1762  std:  0.2601\n",
      "======> DreamBoothLora : \n",
      "Length:  150\n",
      "SC | mean:  0.1213  std:  0.2184\n",
      "PQ | mean:  0.65  std:  0.1375\n",
      "O | mean:  0.1532  std:  0.2279\n",
      "======> BLIPDiffusion_Gen : \n",
      "Length:  150\n",
      "SC | mean:  0.078  std:  0.2036\n",
      "PQ | mean:  0.71  std:  0.1432\n",
      "O | mean:  0.0998  std:  0.2143\n",
      "======> DreamBooth : \n",
      "Length:  150\n",
      "SC | mean:  0.2047  std:  0.3315\n",
      "PQ | mean:  0.7427  std:  0.1256\n",
      "O | mean:  0.2238  std:  0.3123\n",
      "===============> ImagenHub_Subject-Driven_IE\n",
      "======> DreamEdit : \n",
      "Length:  154\n",
      "SC | mean:  0.4188  std:  0.2241\n",
      "PQ | mean:  0.5753  std:  0.1393\n",
      "O | mean:  0.4647  std:  0.1499\n",
      "======> BLIPDiffusion_Edit : \n",
      "Length:  154\n",
      "SC | mean:  0.3305  std:  0.2644\n",
      "PQ | mean:  0.6688  std:  0.163\n",
      "O | mean:  0.4237  std:  0.2119\n",
      "======> PhotoSwap : \n",
      "Length:  154\n",
      "SC | mean:  0.3831  std:  0.2304\n",
      "PQ | mean:  0.6831  std:  0.1445\n",
      "O | mean:  0.4818  std:  0.1754\n",
      "===============> ImagenHub_Text-Guided_IG\n",
      "======> SDXL : \n",
      "Length:  197\n",
      "SC | mean:  0.6061  std:  0.332\n",
      "PQ | mean:  0.6234  std:  0.1369\n",
      "O | mean:  0.5803  std:  0.2335\n",
      "======> SD : \n",
      "Length:  196\n",
      "SC | mean:  0.5026  std:  0.2994\n",
      "PQ | mean:  0.5163  std:  0.1655\n",
      "O | mean:  0.4802  std:  0.1996\n",
      "======> OpenJourney : \n",
      "Length:  197\n",
      "SC | mean:  0.4604  std:  0.3224\n",
      "PQ | mean:  0.5685  std:  0.1271\n",
      "O | mean:  0.4686  std:  0.2201\n",
      "======> DeepFloydIF : \n",
      "Length:  197\n",
      "SC | mean:  0.5853  std:  0.3153\n",
      "PQ | mean:  0.5452  std:  0.1496\n",
      "O | mean:  0.5328  std:  0.2178\n",
      "======> DALLE2 : \n",
      "Length:  196\n",
      "SC | mean:  0.5362  std:  0.2929\n",
      "PQ | mean:  0.5367  std:  0.1644\n",
      "O | mean:  0.5056  std:  0.1896\n"
     ]
    }
   ],
   "source": [
    "iterate_all_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b542bbf9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#analyze_scores()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de61bbad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "393891f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def grab_dataframes(root_dir = '.'):\n",
    "    \"\"\"\n",
    "    Grab dataframes from the subdirectories of a given root directory.\n",
    "\n",
    "    Args:\n",
    "        root_dir (str): The root directory to start searching for .tsv files. Defaults to the current directory.\n",
    "\n",
    "    Returns:\n",
    "        dict: A dictionary where keys are subdirectory names and values are lists of TSV dataframes.\n",
    "    \"\"\"\n",
    "    # Initialize an empty dictionary to store dataframes\n",
    "    dataframes_dict = {}\n",
    "\n",
    "    # List all subdirectories in the root directory\n",
    "    subdirectories = [d for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d))]\n",
    "\n",
    "    # Iterate through each subdirectory\n",
    "    for subdir in subdirectories:\n",
    "        subdir_path = os.path.join(root_dir, subdir)\n",
    "\n",
    "        # Forget about ipynb things\n",
    "        if subdir == \".ipynb_checkpoints\":\n",
    "            continue\n",
    "        # Initialize a list for dataframes in this directory\n",
    "        dir_dataframes = []\n",
    "\n",
    "        # Walk through the current subdirectory\n",
    "        for root, _, files in os.walk(subdir_path):\n",
    "            for file in files:\n",
    "                # Check if the file has a .tsv extension\n",
    "                if file.endswith('.tsv'):\n",
    "                    # Create the full path to the TSV file\n",
    "                    tsv_path = os.path.join(root, file)\n",
    "\n",
    "                    # Read the TSV file into a pandas dataframe\n",
    "                    df = pd.read_csv(tsv_path, sep='\\t')\n",
    "\n",
    "                    # Append the dataframe to the list for this directory\n",
    "                    dir_dataframes.append(df)\n",
    "\n",
    "        # Add the list of dataframes to the dictionary with the subdirectory name as the key\n",
    "        dataframes_dict[subdir] = dir_dataframes\n",
    "\n",
    "    # Now you have a dictionary where keys are subdirectory names and values are lists of TSV dataframes\n",
    "    return dataframes_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2a7d6b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_values_based_on_keys(target_dict, key_reference_dict):\n",
    "    \"\"\"\n",
    "    Extracts values from the target dictionary based on the keys of the reference dictionary.\n",
    "\n",
    "    Parameters:\n",
    "    key_reference_dict (dict): The dictionary whose keys are used as a reference.\n",
    "    target_dict (dict): The dictionary from which values are extracted.\n",
    "\n",
    "    Returns:\n",
    "    dict: A dictionary with keys from the reference dictionary and corresponding values from the target dictionary.\n",
    "    \"\"\"\n",
    "    result = {key: target_dict[key] for key in key_reference_dict.keys() if key in target_dict}\n",
    "    assert list(result.keys()) == list(key_reference_dict.keys())\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00602a09",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_list_of_lists(input_list):\n",
    "    \"\"\"\n",
    "    Convert a mixed list of string representations of lists and actual lists into a list of lists.\n",
    "    \n",
    "    Args:\n",
    "    input_list (list): A list containing a mix of string representations of lists and actual lists.\n",
    "\n",
    "    Returns:\n",
    "    list: A list of lists.\n",
    "    \"\"\"\n",
    "    output_list = []\n",
    "\n",
    "    for item in input_list:\n",
    "        if isinstance(item, str):\n",
    "            # Convert string representation of a list to an actual list\n",
    "            try:\n",
    "                converted_item = eval(item)\n",
    "                if isinstance(converted_item, list):\n",
    "                    output_list.append(converted_item)\n",
    "                else:\n",
    "                    raise ValueError(f\"Item '{item}' is not a valid list representation.\")\n",
    "            except:\n",
    "                raise ValueError(f\"Unable to convert item '{item}' to a list.\")\n",
    "        elif isinstance(item, list):\n",
    "            # Directly append if it's already a list\n",
    "            output_list.append(item)\n",
    "        else:\n",
    "            raise ValueError(f\"Invalid item type: {type(item)}. Expected a string or a list.\")\n",
    "\n",
    "    return output_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "250b8c0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_human_score_raw(list_of_string_lists, cell = 0, overall_mode=False):\n",
    "    \"\"\"\n",
    "    Converts a list of string representations of lists into actual lists, \n",
    "    and then returns the first item from each of these lists.\n",
    "\n",
    "    Parameters:\n",
    "    list_of_string_lists (list of str): A list containing string representations of lists.\n",
    "\n",
    "    Returns:\n",
    "    list: A list of first items from each converted list in the input list.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    list_of_list = convert_to_list_of_lists(list_of_string_lists)\n",
    "    for value_list in list_of_list:\n",
    "        if not overall_mode:\n",
    "            result.append(value_list[cell])\n",
    "        else:\n",
    "            result.append(math.sqrt(value_list[0] * value_list[1]))\n",
    "    return result\n",
    "\n",
    "def combine_gpt4v_human_scores(gpt4v_dict, human_dict, mode=\"SC\"):\n",
    "    gpt4v_l = []\n",
    "    human_l = []\n",
    "    o_human_l = []\n",
    "    \n",
    "    if mode == \"O\":\n",
    "        for key in gpt4v_dict:\n",
    "            human_score_raw = human_dict[key]\n",
    "            #print(human_score_raw)\n",
    "            o_human_score = extract_human_score_raw(human_score_raw, overall_mode=True)\n",
    "            o_human_score = np.mean(o_human_score)\n",
    "            o_human_l.append(o_human_score)\n",
    "        return o_human_l\n",
    "    \n",
    "    for key in gpt4v_dict:\n",
    "        if (mode == \"SC\"):\n",
    "            cell = 0\n",
    "        elif (mode == \"PQ\"):\n",
    "            cell = 1\n",
    "        gpt4v_score_raw = gpt4v_dict[key]\n",
    "        if isinstance(gpt4v_score_raw, (int, float)):\n",
    "            gpt4v_score = map_to_nearest_higher(gpt4v_score_raw/10.0)\n",
    "        else:\n",
    "            gpt4v_score_raw = [int(score) for score in gpt4v_score_raw]\n",
    "            gpt4v_score = map_to_nearest_higher(min(gpt4v_score_raw)/10.0)\n",
    "        human_score_raw = human_dict[key]\n",
    "        #print(human_score_raw)\n",
    "        human_score = extract_human_score_raw(human_score_raw, cell)\n",
    "        human_score = np.mean(human_score)\n",
    "        gpt4v_l.append(gpt4v_score)\n",
    "        human_l.append(human_score)\n",
    "    return gpt4v_l, human_l\n",
    "\n",
    "def dict_to_value_list(dictionary):\n",
    "    \"\"\"\n",
    "    Convert a dictionary into a list of its values.\n",
    "\n",
    "    Args:\n",
    "    dictionary (dict): A dictionary from which values are to be extracted.\n",
    "\n",
    "    Returns:\n",
    "    list: A list containing the values of the dictionary.\n",
    "    \"\"\"\n",
    "    return list(dictionary.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3ded8858",
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_correlation(z_scores):\n",
    "    \"\"\"\n",
    "    Averages a list of Fisher Z-transformed correlation scores and converts it back to a correlation coefficient.\n",
    "\n",
    "    :param z_scores: A list of Fisher Z-transformed correlation scores.\n",
    "    :return: The averaged correlation coefficient.\n",
    "    \"\"\"\n",
    "    # Calculate the average Z score\n",
    "    z_avg = sum(z_scores) / len(z_scores)\n",
    "\n",
    "    # Convert the average Z score back to a correlation coefficient\n",
    "    r_avg = (math.exp(2 * z_avg) - 1) / (math.exp(2 * z_avg) + 1)\n",
    "    return r_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "99ce6b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterate_all_results_corr(root=ROOT, use_spearmanr=1):\n",
    "    task_list = os.listdir(root) \n",
    "    all_task_SC_correlation = []\n",
    "    all_task_PQ_correlation = []\n",
    "    all_task_O_correlation = []\n",
    "    for task in task_list:\n",
    "        print(\"===============>\", task)\n",
    "        model_list = os.listdir(os.path.join(root,task))\n",
    "        human_data = grab_dataframes(HUMAN_RESULTS)[task]\n",
    "        \n",
    "        human_result = {model: {} for model in model_list}\n",
    "        for df in human_data:\n",
    "            for index, row in df.iterrows():\n",
    "                uid = row['uid']\n",
    "\n",
    "                for model in human_result.keys():\n",
    "                    if uid not in human_result[model]:\n",
    "                        human_result[model][uid] = []\n",
    "\n",
    "                    human_result[model][uid].append(row[model])\n",
    "        \n",
    "        task_SC_correlation = []\n",
    "        task_PQ_correlation = []\n",
    "        task_O_correlation = []\n",
    "        for model in model_list:\n",
    "            print(\"======>\", model, \": \")\n",
    "            SC_results = grab_gpt4v_processed(task, model, \"SC\")\n",
    "            PQ_results = grab_gpt4v_processed(task, model, \"PQ\")\n",
    "            \n",
    "            SC_results, PQ_results = align_dicts(SC_results, PQ_results)\n",
    "            O_results = get_O_from_two_dicts_gpt4v(SC_results, PQ_results)\n",
    "            \n",
    "            SC_results_human = extract_values_based_on_keys(human_result[model], SC_results)\n",
    "            SC_gpt4v, SC_human = combine_gpt4v_human_scores(SC_results, SC_results_human, mode=\"SC\")\n",
    "            \n",
    "\n",
    "            PQ_results_human = extract_values_based_on_keys(human_result[model], PQ_results)\n",
    "            PQ_gpt4v, PQ_human = combine_gpt4v_human_scores(PQ_results, PQ_results_human, mode=\"PQ\")\n",
    "\n",
    "            O_results_human = PQ_results_human # Basically same source\n",
    "            O_human = combine_gpt4v_human_scores(O_results, O_results_human, mode=\"O\")\n",
    "            O_gpt4v = dict_to_value_list(O_results)\n",
    "            \n",
    "            if use_spearmanr==2: \n",
    "                SC_rho, _ = kendalltau(SC_gpt4v, SC_human)\n",
    "                print(\"SC|\", sigfig(SC_rho))\n",
    "                PQ_rho, _ = kendalltau(PQ_gpt4v, PQ_human)\n",
    "                print(\"PQ|\", sigfig(PQ_rho))\n",
    "                O_rho, _ = kendalltau(O_gpt4v, O_human)\n",
    "                print(\"O|\", sigfig(O_rho))\n",
    "            elif use_spearmanr==1:\n",
    "                SC_rho, _ = spearmanr(SC_gpt4v, SC_human)\n",
    "                print(\"SC|\", sigfig(SC_rho))\n",
    "                PQ_rho, _ = spearmanr(PQ_gpt4v, PQ_human)\n",
    "                print(\"PQ|\", sigfig(PQ_rho))\n",
    "                O_rho, _ = spearmanr(O_gpt4v, O_human)\n",
    "                print(\"O|\", sigfig(O_rho))\n",
    "            elif use_spearmanr==0:\n",
    "                SC_rho = np.corrcoef(SC_gpt4v, SC_human)[0, 1]\n",
    "                print(\"SC|\", sigfig(SC_rho))\n",
    "                PQ_rho = np.corrcoef(PQ_gpt4v, PQ_human)[0, 1]\n",
    "                print(\"PQ|\", sigfig(PQ_rho))\n",
    "                O_rho = np.corrcoef(O_gpt4v, O_human)[0, 1]\n",
    "                print(\"O|\", sigfig(O_rho))\n",
    "            task_SC_correlation.append(SC_rho)\n",
    "            task_PQ_correlation.append(PQ_rho)\n",
    "            task_O_correlation.append(O_rho)\n",
    "        print(f\"++++++++++++++> {task} Avg\")\n",
    "        task_SC_correlation_avg = average_correlation(task_SC_correlation)\n",
    "        task_PQ_correlation_avg = average_correlation(task_PQ_correlation)\n",
    "        task_O_correlation_avg = average_correlation(task_O_correlation)\n",
    "        print(task,\"|SC|\",sigfig(task_SC_correlation_avg))\n",
    "        print(task,\"|PQ|\",sigfig(task_PQ_correlation_avg))\n",
    "        print(task,\"|O|\",sigfig(task_O_correlation_avg))\n",
    "        all_task_SC_correlation.append(task_SC_correlation_avg)\n",
    "        all_task_PQ_correlation.append(task_PQ_correlation_avg)\n",
    "        all_task_O_correlation.append(task_O_correlation_avg)\n",
    "\n",
    "    final_SC_corr_avg = average_correlation(all_task_SC_correlation)\n",
    "    final_PQ_corr_avg = average_correlation(all_task_PQ_correlation)\n",
    "    final_O_corr_avg = average_correlation(all_task_O_correlation)\n",
    "    print(\"\")\n",
    "    print(\"VVVVVVVVVVVVVVVVVV> FINAL Correlation Across 7 Tasks\")\n",
    "    print(\"|SC|\",sigfig(final_SC_corr_avg))\n",
    "    print(\"|PQ|\",sigfig(final_PQ_corr_avg))\n",
    "    print(\"|O|\",sigfig(final_O_corr_avg))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "37e59976",
   "metadata": {},
   "outputs": [],
   "source": [
    "#iterate_all_results_corr(use_spearmanr=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "14e09bd6-6beb-4512-82f4-b774c932b873",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============> ImagenHub_Multi-Concept_IC\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======> TextualInversion : \n",
      "SC| 0.3538\n",
      "PQ| -0.0053\n",
      "O| 0.3134\n",
      "======> DreamBooth : \n",
      "SC| 0.2145\n",
      "PQ| 0.2928\n",
      "O| 0.1707\n",
      "======> CustomDiffusion : \n",
      "SC| 0.5476\n",
      "PQ| 0.3046\n",
      "O| 0.5491\n",
      "++++++++++++++> ImagenHub_Multi-Concept_IC Avg\n",
      "ImagenHub_Multi-Concept_IC |SC| 0.3557\n",
      "ImagenHub_Multi-Concept_IC |PQ| 0.1948\n",
      "ImagenHub_Multi-Concept_IC |O| 0.3314\n",
      "===============> ImagenHub_Text-Guided_IE\n",
      "======> Prompt2prompt : \n",
      "SC| 0.3249\n",
      "PQ| 0.3611\n",
      "O| 0.3268\n",
      "======> CycleDiffusion : \n",
      "SC| 0.4387\n",
      "PQ| 0.5941\n",
      "O| 0.4009\n",
      "======> SDEdit : \n",
      "SC| 0.1091\n",
      "PQ| 0.4078\n",
      "O| 0.1077\n",
      "======> DiffEdit : \n",
      "SC| 0.1719\n",
      "PQ| 0.5632\n",
      "O| 0.0947\n",
      "======> MagicBrush : \n",
      "SC| 0.4689\n",
      "PQ| 0.4927\n",
      "O| 0.4727\n",
      "======> Text2Live : \n",
      "SC| 0.1368\n",
      "PQ| 0.4136\n",
      "O| 0.1356\n",
      "======> Pix2PixZero : \n",
      "SC| 0.197\n",
      "PQ| 0.4698\n",
      "O| 0.2188\n",
      "======> InstructPix2Pix : \n",
      "SC| 0.4856\n",
      "PQ| 0.3684\n",
      "O| 0.4822\n",
      "++++++++++++++> ImagenHub_Text-Guided_IE Avg\n",
      "ImagenHub_Text-Guided_IE |SC| 0.2836\n",
      "ImagenHub_Text-Guided_IE |PQ| 0.4291\n",
      "ImagenHub_Text-Guided_IE |O| 0.2728\n",
      "===============> ImagenHub_Control-Guided_IG\n",
      "======> UniControl : \n",
      "SC| 0.4059\n",
      "PQ| 0.3143\n",
      "O| 0.3277\n",
      "======> ControlNet : \n",
      "SC| 0.2695\n",
      "PQ| 0.3846\n",
      "O| 0.2819\n",
      "++++++++++++++> ImagenHub_Control-Guided_IG Avg\n",
      "ImagenHub_Control-Guided_IG |SC| 0.3254\n",
      "ImagenHub_Control-Guided_IG |PQ| 0.3359\n",
      "ImagenHub_Control-Guided_IG |O| 0.2957\n",
      "===============> ImagenHub_Mask-Guided_IE\n",
      "======> Glide : \n",
      "SC| 0.4417\n",
      "PQ| 0.3575\n",
      "O| 0.3965\n",
      "======> BlendedDiffusion : \n",
      "SC| 0.2487\n",
      "PQ| 0.224\n",
      "O| 0.1154\n",
      "======> SDInpaint : \n",
      "SC| 0.5325\n",
      "PQ| 0.3008\n",
      "O| 0.4443\n",
      "======> SDXLInpaint : \n",
      "SC| 0.6185\n",
      "PQ| 0.2853\n",
      "O| 0.5481\n",
      "++++++++++++++> ImagenHub_Mask-Guided_IE Avg\n",
      "ImagenHub_Mask-Guided_IE |SC| 0.4304\n",
      "ImagenHub_Mask-Guided_IE |PQ| 0.2839\n",
      "ImagenHub_Mask-Guided_IE |O| 0.3593\n",
      "===============> ImagenHub_Subject-Driven_IG\n",
      "======> TextualInversion : \n",
      "SC| 0.5521\n",
      "PQ| 0.2332\n",
      "O| 0.5356\n",
      "======> DreamBoothLora : \n",
      "SC| 0.2051\n",
      "PQ| 0.245\n",
      "O| 0.2093\n",
      "======> BLIPDiffusion_Gen : \n",
      "SC| 0.121\n",
      "PQ| -0.0779\n",
      "O| 0.1214\n",
      "======> DreamBooth : \n",
      "SC| 0.3185\n",
      "PQ| 0.3132\n",
      "O| 0.3062\n",
      "++++++++++++++> ImagenHub_Subject-Driven_IG Avg\n",
      "ImagenHub_Subject-Driven_IG |SC| 0.2906\n",
      "ImagenHub_Subject-Driven_IG |PQ| 0.1765\n",
      "ImagenHub_Subject-Driven_IG |O| 0.285\n",
      "===============> ImagenHub_Subject-Driven_IE\n",
      "======> DreamEdit : \n",
      "SC| 0.3079\n",
      "PQ| 0.3328\n",
      "O| 0.2327\n",
      "======> BLIPDiffusion_Edit : \n",
      "SC| 0.1841\n",
      "PQ| 0.3174\n",
      "O| 0.2293\n",
      "======> PhotoSwap : \n",
      "SC| 0.175\n",
      "PQ| 0.3274\n",
      "O| 0.2198\n",
      "++++++++++++++> ImagenHub_Subject-Driven_IE Avg\n",
      "ImagenHub_Subject-Driven_IE |SC| 0.2187\n",
      "ImagenHub_Subject-Driven_IE |PQ| 0.3148\n",
      "ImagenHub_Subject-Driven_IE |O| 0.2234\n",
      "===============> ImagenHub_Text-Guided_IG\n",
      "======> SDXL : \n",
      "SC| 0.5974\n",
      "PQ| 0.292\n",
      "O| 0.5505\n",
      "======> SD : \n",
      "SC| 0.6104\n",
      "PQ| 0.1879\n",
      "O| 0.4544\n",
      "======> OpenJourney : \n",
      "SC| 0.5904\n",
      "PQ| 0.2523\n",
      "O| 0.5538\n",
      "======> DeepFloydIF : \n",
      "SC| 0.5528\n",
      "PQ| 0.0659\n",
      "O| 0.4361\n",
      "======> DALLE2 : \n",
      "SC| 0.4783\n",
      "PQ| 0.1336\n",
      "O| 0.3412\n",
      "++++++++++++++> ImagenHub_Text-Guided_IG Avg\n",
      "ImagenHub_Text-Guided_IG |SC| 0.5123\n",
      "ImagenHub_Text-Guided_IG |PQ| 0.1842\n",
      "ImagenHub_Text-Guided_IG |O| 0.4359\n",
      "\n",
      "VVVVVVVVVVVVVVVVVV> FINAL Correlation Across 7 Tasks\n",
      "|SC| 0.3322\n",
      "|PQ| 0.2675\n",
      "|O| 0.3048\n"
     ]
    }
   ],
   "source": [
    "iterate_all_results_corr(use_spearmanr=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "757be891",
   "metadata": {},
   "outputs": [],
   "source": [
    "#iterate_all_results_corr(use_spearmanr=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "22c0097a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_spearman_correlation_by_item(data, target_user, use_spearmanr=1):\n",
    "    \"\"\"\n",
    "    Computes the Spearman rank correlation of the target user's ratings for each item (A and B) \n",
    "    with the average ratings of the other two users for the same items.\n",
    "\n",
    "    :param data: A dictionary with keys as image names and values as a list of rating pairs [A, B].\n",
    "    :param target_user: Index (0, 1, or 2) of the target user whose ratings are to be compared.\n",
    "    :return: A dictionary containing Spearman rank correlation coefficients and p-values for each item.\n",
    "    \"\"\"\n",
    "    target_ratings_SC = []\n",
    "    target_ratings_PQ = []\n",
    "    target_ratings_O = []\n",
    "    avg_other_ratings_SC = []\n",
    "    avg_other_ratings_PQ = []\n",
    "    avg_other_ratings_O = []\n",
    "\n",
    "    for ratings in data.values():\n",
    "        # Extract the ratings of the target user and the other two users\n",
    "        target_rating = ratings[target_user]\n",
    "        other_ratings = [ratings[i] for i in range(3) if i != target_user]\n",
    "\n",
    "        # Convert string ratings to list of floats\n",
    "\n",
    "        O_ratings = [list(map(float, r.strip('[]').split(','))) for r in ratings]\n",
    "        O_ratings = [math.sqrt(r[0] * r[1]) for r in O_ratings]\n",
    "\n",
    "        target_rating = list(map(float, target_rating.strip('[]').split(',')))\n",
    "        other_ratings = [list(map(float, r.strip('[]').split(','))) for r in other_ratings]\n",
    "\n",
    "        O_target_rating = O_ratings[target_user]\n",
    "        O_other_ratings = [O_ratings[i] for i in range(3) if i != target_user]\n",
    "\n",
    "        # Calculate the average of the other two users' ratings\n",
    "        avg_rating = np.mean(other_ratings, axis=0)\n",
    "\n",
    "        O_avg_rating = np.mean(O_other_ratings, axis=0)\n",
    "\n",
    "        # Append the ratings for each item for correlation calculation\n",
    "        target_ratings_SC.append(target_rating[0])\n",
    "        target_ratings_PQ.append(target_rating[1])\n",
    "        target_ratings_O.append(O_target_rating)\n",
    "        avg_other_ratings_SC.append(avg_rating[0])\n",
    "        avg_other_ratings_PQ.append(avg_rating[1])\n",
    "        avg_other_ratings_O.append(O_avg_rating)\n",
    "        \n",
    "    if use_spearmanr==2:\n",
    "        correlation_SC, p_value_SC = kendalltau(target_ratings_SC, avg_other_ratings_SC)\n",
    "        correlation_PQ, p_value_PQ = kendalltau(target_ratings_PQ, avg_other_ratings_PQ)\n",
    "        correlation_O, p_value_O = kendalltau(target_ratings_O, avg_other_ratings_O)\n",
    "        return {\n",
    "            'SC': {'Correlation': correlation_SC, 'P-Value': p_value_SC},\n",
    "            'PQ': {'Correlation': correlation_PQ, 'P-Value': p_value_PQ},\n",
    "            'O': {'Correlation': correlation_O, 'P-Value': p_value_O},\n",
    "        }\n",
    "    elif use_spearmanr==1:\n",
    "        correlation_SC, p_value_SC = spearmanr(target_ratings_SC, avg_other_ratings_SC)\n",
    "        correlation_PQ, p_value_PQ = spearmanr(target_ratings_PQ, avg_other_ratings_PQ)\n",
    "        correlation_O, p_value_O = spearmanr(target_ratings_O, avg_other_ratings_O)\n",
    "        return {\n",
    "            'SC': {'Correlation': correlation_SC, 'P-Value': p_value_SC},\n",
    "            'PQ': {'Correlation': correlation_PQ, 'P-Value': p_value_PQ},\n",
    "            'O': {'Correlation': correlation_O, 'P-Value': p_value_O},\n",
    "        }\n",
    "    elif use_spearmanr==0:\n",
    "        correlation_SC = np.corrcoef(target_ratings_SC, avg_other_ratings_SC)[0, 1]\n",
    "        correlation_PQ = np.corrcoef(target_ratings_PQ, avg_other_ratings_PQ)[0, 1]\n",
    "        correlation_O = np.corrcoef(target_ratings_O, avg_other_ratings_O)[0, 1]\n",
    "        return {\n",
    "            'SC': {'Correlation': correlation_SC},\n",
    "            'PQ': {'Correlation': correlation_PQ},\n",
    "            'O': {'Correlation': correlation_O},\n",
    "        }\n",
    "\n",
    "        \n",
    "\n",
    "def iterate_all_results_human2human(root=ROOT, target_user_index=0, num_raters=3, use_spearmanr=1):\n",
    "    task_list = os.listdir(root) \n",
    "    for task in task_list:\n",
    "        print(\"===============>\", task)\n",
    "        model_list = os.listdir(os.path.join(root,task))\n",
    "        human_data = grab_dataframes(HUMAN_RESULTS)[task]\n",
    "        \n",
    "        human_result = {model: {} for model in model_list}\n",
    "        for df in human_data:\n",
    "            for index, row in df.iterrows():\n",
    "                uid = row['uid']\n",
    "\n",
    "                for model in human_result.keys():\n",
    "                    if uid not in human_result[model]:\n",
    "                        human_result[model][uid] = []\n",
    "\n",
    "                    human_result[model][uid].append(row[model])\n",
    "        for model in model_list:\n",
    "            print(\"======>\", model, \": \")\n",
    "            data = human_result[model]\n",
    "            corr_list_SC = []\n",
    "            corr_list_PQ = []\n",
    "            corr_list_O = []\n",
    "            for i in range(num_raters):\n",
    "                human_corr_dict = compute_spearman_correlation_by_item(data, i, use_spearmanr=use_spearmanr)\n",
    "                corr_list_SC.append(human_corr_dict['SC']['Correlation'])\n",
    "                corr_list_PQ.append(human_corr_dict['PQ']['Correlation'])\n",
    "                corr_list_O.append(human_corr_dict['O']['Correlation'])\n",
    "            print(f\"Human2Human for SC:\", sigfig(average_correlation(corr_list_SC)))\n",
    "            print(f\"Human2Human for PQ:\", sigfig(average_correlation(corr_list_PQ)))\n",
    "            print(f\"Human2Human for O:\", sigfig(average_correlation(corr_list_O)))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "aa920506-7ed2-480c-ab9c-ad11e12b562c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#iterate_all_results_human2human(use_spearmanr=0) #pearson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a3a0fad7-37e6-4a3a-a2d4-1c160ad290e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============> ImagenHub_Multi-Concept_IC\n",
      "======> TextualInversion : \n",
      "Human2Human for SC: 0.699\n",
      "Human2Human for PQ: 0.5803\n",
      "Human2Human for O: 0.698\n",
      "======> DreamBooth : \n",
      "Human2Human for SC: 0.6209\n",
      "Human2Human for PQ: 0.6423\n",
      "Human2Human for O: 0.6222\n",
      "======> CustomDiffusion : \n",
      "Human2Human for SC: 0.7256\n",
      "Human2Human for PQ: 0.4838\n",
      "Human2Human for O: 0.7217\n",
      "===============> ImagenHub_Text-Guided_IE\n",
      "======> Prompt2prompt : \n",
      "Human2Human for SC: 0.588\n",
      "Human2Human for PQ: 0.5028\n",
      "Human2Human for O: 0.5811\n",
      "======> CycleDiffusion : \n",
      "Human2Human for SC: 0.5482\n",
      "Human2Human for PQ: 0.5887\n",
      "Human2Human for O: 0.5891\n",
      "======> SDEdit : \n",
      "Human2Human for SC: 0.2657\n",
      "Human2Human for PQ: 0.4705\n",
      "Human2Human for O: 0.1991\n",
      "======> DiffEdit : \n",
      "Human2Human for SC: 0.3286\n",
      "Human2Human for PQ: 0.6214\n",
      "Human2Human for O: 0.4265\n",
      "======> MagicBrush : \n",
      "Human2Human for SC: 0.6219\n",
      "Human2Human for PQ: 0.519\n",
      "Human2Human for O: 0.6289\n",
      "======> Text2Live : \n",
      "Human2Human for SC: 0.2675\n",
      "Human2Human for PQ: 0.5757\n",
      "Human2Human for O: 0.1524\n",
      "======> Pix2PixZero : \n",
      "Human2Human for SC: 0.3311\n",
      "Human2Human for PQ: 0.577\n",
      "Human2Human for O: 0.3327\n",
      "======> InstructPix2Pix : \n",
      "Human2Human for SC: 0.66\n",
      "Human2Human for PQ: 0.5955\n",
      "Human2Human for O: 0.6561\n",
      "===============> ImagenHub_Control-Guided_IG\n",
      "======> UniControl : \n",
      "Human2Human for SC: 0.606\n",
      "Human2Human for PQ: 0.6062\n",
      "Human2Human for O: 0.5954\n",
      "======> ControlNet : \n",
      "Human2Human for SC: 0.6144\n",
      "Human2Human for PQ: 0.5682\n",
      "Human2Human for O: 0.5868\n",
      "===============> ImagenHub_Mask-Guided_IE\n",
      "======> Glide : \n",
      "Human2Human for SC: 0.5894\n",
      "Human2Human for PQ: 0.553\n",
      "Human2Human for O: 0.5695\n",
      "======> BlendedDiffusion : \n",
      "Human2Human for SC: 0.5051\n",
      "Human2Human for PQ: 0.5511\n",
      "Human2Human for O: 0.4224\n",
      "======> SDInpaint : \n",
      "Human2Human for SC: 0.659\n",
      "Human2Human for PQ: 0.5166\n",
      "Human2Human for O: 0.5394\n",
      "======> SDXLInpaint : \n",
      "Human2Human for SC: 0.6574\n",
      "Human2Human for PQ: 0.5928\n",
      "Human2Human for O: 0.6556\n",
      "===============> ImagenHub_Subject-Driven_IG\n",
      "======> TextualInversion : \n",
      "Human2Human for SC: 0.6\n",
      "Human2Human for PQ: 0.3351\n",
      "Human2Human for O: 0.5686\n",
      "======> DreamBoothLora : \n",
      "Human2Human for SC: 0.3903\n",
      "Human2Human for PQ: 0.443\n",
      "Human2Human for O: 0.3878\n",
      "======> BLIPDiffusion_Gen : \n",
      "Human2Human for SC: 0.4458\n",
      "Human2Human for PQ: 0.3263\n",
      "Human2Human for O: 0.439\n",
      "======> DreamBooth : \n",
      "Human2Human for SC: 0.6452\n",
      "Human2Human for PQ: 0.3871\n",
      "Human2Human for O: 0.6208\n",
      "===============> ImagenHub_Subject-Driven_IE\n",
      "======> DreamEdit : \n",
      "Human2Human for SC: 0.5867\n",
      "Human2Human for PQ: 0.2245\n",
      "Human2Human for O: 0.546\n",
      "======> BLIPDiffusion_Edit : \n",
      "Human2Human for SC: 0.5359\n",
      "Human2Human for PQ: 0.4033\n",
      "Human2Human for O: 0.5051\n",
      "======> PhotoSwap : \n",
      "Human2Human for SC: 0.4805\n",
      "Human2Human for PQ: 0.2961\n",
      "Human2Human for O: 0.4973\n",
      "===============> ImagenHub_Text-Guided_IG\n",
      "======> SDXL : \n",
      "Human2Human for SC: 0.5807\n",
      "Human2Human for PQ: 0.4992\n",
      "Human2Human for O: 0.5896\n",
      "======> SD : \n",
      "Human2Human for SC: 0.5979\n",
      "Human2Human for PQ: 0.2772\n",
      "Human2Human for O: 0.4962\n",
      "======> OpenJourney : \n",
      "Human2Human for SC: 0.5321\n",
      "Human2Human for PQ: 0.36\n",
      "Human2Human for O: 0.4861\n",
      "======> DeepFloydIF : \n",
      "Human2Human for SC: 0.5635\n",
      "Human2Human for PQ: 0.3029\n",
      "Human2Human for O: 0.5131\n",
      "======> DALLE2 : \n",
      "Human2Human for SC: 0.5019\n",
      "Human2Human for PQ: 0.468\n",
      "Human2Human for O: 0.4348\n"
     ]
    }
   ],
   "source": [
    "iterate_all_results_human2human(use_spearmanr=1) #spearman"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a0022920-0a44-4076-bdef-5bac44f71502",
   "metadata": {},
   "outputs": [],
   "source": [
    "#iterate_all_results_human2human(use_spearmanr=2) #kendalltau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c8fc06",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe856238-281f-4c9a-b3d9-1e4bced34c82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ca81f3c-a397-4a69-80de-6755a22b968a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
